import requests
import os
import sys
import json
import datetime
import re
import logging
import time
import math
pwd = os.getcwd()
sys.path.append(pwd)
# from proxy import Proxy
from lxml import etree
from urllib.parse import urlencode
from requests.exceptions import ConnectTimeout
# gov数据库
from sql import sql_gov_model

class GovTmri(object):
    url = 'http://www.tmri.cn/list.aspx'
    headers = {
        'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/52.0.2743.116 Safari/537.36',
        'Host': 'www.tmri.cn',
        'Pragma': 'no-cache',
        'Referer': 'http://www.tmri.cn/index.aspx',
    }
    base_dir = './'

    dict_id = { '1018': '通知公告', # 通知公告
                '1002': '工作动态', # 工作动态
                '1017': '专家访谈', # 专家访谈
                '1019': '重点科研项目', # 重点科研项目
                '1005': '交通安全产品检测与质量监督信息通报', # 交通安全产品检测与质量监督信息通报
                '1004': '成果与产品展示', # 成果与产品展示
                '1008': '全国交标委工作动态', # 全国交标委工作动态
                '1010': '出版刊物', # 出版刊物
                '1013': '道路交通安全重点实验室', # 道路交通安全重点实验室
                '1022': '交标委_工作动态', # 交标委_工作动态
                }

    def __init__(self):
        # self.proxy = Proxy().get_proxy()
        self.params = {
            'id': 1001
        }
    # 获取昨天的年月日
    def get_yestoday(self):
        yesterday = datetime.datetime.today() + datetime.timedelta(-1)
        str_yesterday = datetime.datetime.strftime(yesterday, "%Y-%m-%d")
        return str_yesterday

    # 请求网页,获取html
    def request(self):
        for str_id in self.dict_id.keys():
            self.params['id'] = str_id
            response = requests.get(url='https://www.baidu.com/s', headers=self.headers)
            response.status_code = 404
            while response.status_code != 200:
                try:
                    response = requests.get(url=self.url, headers=self.headers, params=self.params, timeout=5)
                except ConnectTimeout:
                    print(str_id, '超时', response.status_code, 'sleep(120)')
                    time.sleep(120)
                    # response = requests.get(url=self.url, headers=self.headers, params=self.params, timeout=5)
            response.encoding = 'gb2312'
            self.response = response.text
            # 解析页面
            self.page_parse(str_id)

    # 获取最大页数
    def get_max_page(self):
        try:
            result = re.search('共(\d+)页', self.response).group(1)
            max_page = int(result)
        except AttributeError as e:
            max_page = 0
            print(self.data['key'], '搜索出错')
        return max_page
    
    # 获取招标信息
    def page_parse(self, str_id):
        html = etree.HTML(self.response)
        list_content = html.xpath('//*[@id="html_li"]/ul/li') # 一页的所有记录
        # 如果list_content长度为0,说明当前页没有一条记录
        if len(list_content) == 0:
            return None
        # list_content[0]：标题，时间
        for current_content in list_content[1:]:
            title = current_content.xpath('a/@title')[0] # 标题
            # try:
            result = re.search('人工智能|人才|职称|项目|专项|指南|申报|奖励', title)
            if result != None:
                print(result.group(), title)
            else:
                print('没找到')
                continue

            url = 'http://www.tmri.cn/'+current_content.xpath('a/@href')[0] # 链接

            str_time = current_content.xpath('span/text()')[0]
            str_time = datetime.datetime.strptime(str_time, '%Y-%m-%d') # 时间

            print(url, title, str_time)
            gov = sql_gov_model.Gov(website='公安部交通管理科学研究所', keyword=result.group(), title=title, url=url, str_time=str_time, str_type=self.dict_id[str_id], desc='')
            sql_gov_model.insert_gov(gov)

    # 向文件写内容
    def write_file(self, content):
        with open(self.save_file, 'a') as f:
            f.write(content)

    def folder_exist(self, dir_path):
        '''
        1. 作用:判断文件夹路径是否存在,不存在则创建
        2. 参数:dir_path:文件夹路径
        3. 返回值:None
        '''
        if not os.path.exists(dir_path):
            os.makedirs(dir_path)

if __name__ == '__main__':
    gov_tmri = GovTmri()
    gov_tmri.request()

